/**
 * author:Administrator
 * date:2016/4/12
 * desc:从 www.51voa.com上抓取 MP3文件
 * 正常工作
 */
'use strict';

// lib
var moment = require('moment');
var request = require('request');
var urllib = require('url');
var cheerio = require('cheerio');
var fs = require('fs');
var path = require('path');

// const
var ROOT_HOST = 'http://www.51voa.com';
var MAX_DEPTH = 2;
var FILE_PREFIX = 'mp3';
var WHITE_LIST = ['html', 'mp3']; // 之后抓取这些文件
var folder = moment(new Date()).format('YYYY-MM-DD');

//
var urlList = [];       // 尚未爬取
var filterList = [];    // 已经爬取过 或 已在爬取列表中
var mp3UrlList = [];    // 下载列表

function main() {
  console.log('main');
  urlList.push({url: ROOT_HOST, depth: 0});
  filterList.push(ROOT_HOST);

  crawlerPageForMp3List(function () {
    console.log('result mp3list =', mp3UrlList);

    downMp3(function () {
      console.log('---------all done-------');
    });
  });
}

function downMp3(cb) {
  // mkdir
  var dir = path.join(__dirname, folder);
  try {
    var stats = fs.lstatSync(folder);
    if (stats.isDirectory()) {
    }
  } catch (e) {
    console.log('dir %s not exists make it', folder);
    fs.mkdirSync(folder, 0o777);
  }

  function oneByOne() {
    if (mp3UrlList.length == 0) return cb && cb();
    var url = mp3UrlList.shift();
    var name = url.split('/').pop();
    var filePath = path.join(dir, name);
    console.log('download, file=', filePath);
    var file = fs.createWriteStream(filePath);
    request(url).pipe(file);
    file.on('finish', function () {
      console.log('finish:', filePath);
      file.close(oneByOne);  // close() is async, call cb after close completes.
    });
  }

  oneByOne();
}

/**
 * 抓取网页，获取MP3列表
 */
function crawlerPageForMp3List(cb) {
  console.log('crawlerPageForMp3List');
  var called = false;

  function loop() {
    if (urlList.length == 0) {
      if (!called) {
        called = true;
        cb && cb();
      }
      return;
    }

    // console.log('urlList remain', urlList.length);
    var item = urlList.shift();
    if (item.depth == MAX_DEPTH) {       // reach max depth
      loop();
      return;
    }
    console.log('--------------------------------');
    console.log(item.url);
    requestHtml(item, function (list) {
      handleUrls(list);
      loop();
    });
  }

  loop();
}

/**
 * @param {Array}   urls
 */
function handleUrls(urls) {
  urls.forEach(function (item) {
    // console.log('depth=%s,url=%s', item.depth, item.url);
    var url = item.url;
    var postfix = url.split('.').pop();
    if (WHITE_LIST.indexOf(postfix) >= 0 && filterList.indexOf(url) < 0 && item.depth <= MAX_DEPTH) {
      if (url.endsWith(FILE_PREFIX)) {
        console.log('add mp3', url);
        mp3UrlList.push(url);
      } else {
        // console.log('add ', url);
        filterList.push(url);
        urlList.push(item);
      }
    }
  });
}

/**
 * 抓取网页，并分析，返回a标签列表
 * @param item
 * @param cb
 */
function requestHtml(item, cb) {
  console.log('requestHtml', item.depth, item.url);
  if (item.depth > MAX_DEPTH) return cb && cb([]);
  var url = item.url;
  if (!url) return cb && cb([]);
  request(url, function (err, res, body) {
    if (err) {
      console.error('request error,url=%s', url, err);
      return cb && cb([]);
    }

    var $ = cheerio.load(body);
    var allHref = [];
    $('a').each(function () {
      var href = $(this).attr('href');
      if (href) {
        var fulUrl = urllib.resolve(url, href);
        allHref.push(fulUrl);
      }
    });

    var list = allHref.map(aUrl => {
      return {url: aUrl, depth: item.depth + 1}
    });
    cb && cb(list);
  });
}

main();